import requests
from lxml import etree
import urllib
from bs4 import BeautifulSoup
# from Buyer.models import *
import random
import time


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
}
types = ['进口奶粉', '宝宝辅食', '营养品', '纸尿裤', '婴儿湿巾',
         '婴儿车', '婴儿床', '儿童安全座椅', '儿童玩具', '早教书籍', '孕产育儿书',
         '童装', '童鞋', '宝宝配饰', '孕妇装', '孕妇护肤', '孕妇用品']
# location = ['四川', '深圳', '杭州', '北京', '江苏', '云南', '山东', '广东', '江西', '河南', '河北', '湖南', '湖北', '山西', '陕西']
# store = ['好佳佳母婴店', '亲亲宝贝', '宝宝最爱', '妈妈咪呀', '嗨乐购']
for type in types:
    for i in range(1, 50):
        url = 'http://s.leyou.com.cn/search?key=%E8%BF%9B%E5%8F%A3%E5%A5%B6%E7%B2%89&page='
        url = url + str(i)
        print(url)
        content = requests.get(url, headers=headers).content.decode('utf-8')
        with open('进口奶粉-{}.txt'.format(i), mode='w', encoding='utf-8') as fp:
            fp.write(content)
        tree = etree.HTML(content)
        # soup = BeautifulSoup(content, 'html.parser')
        # print(soup.select('img'))
        # with open('进口奶粉.txt', mode='a', encoding='utf-8') as fp:
        #     fp.write(str(soup))
        # print('00000')
        div_list = tree.xpath(r'//div[@class="wrapperList"]/div[@class="products"]')
        # print(len(div_list))
        if len(div_list) > 0:
            goods = {}
            # print('11111')
            for div in div_list:
                src = div.xpath(r'./span[@class="pic"]/a/img/@src')[0]
                # print(src)
                href = div.xpath(r'./span[@class="pic"]/a/@href')[0]
                # print(href)
                price = div.xpath(r'./span[@class="disPrice"]/text()')[0].replace('￥', '')
                title = div.xpath(r'./span[@class="intro"]/a/@title')[0]
                goods['src'] = src
                goods['href'] = href
                goods['price'] = price
                goods['title'] = title
                with open('进口奶粉--1.txt', mode='a', encoding='utf-8') as fp:
                    fp.write(str(goods) + '\n')
        else:
            continue


